#-*- coding:utf-8 -*-

import scrapy
import datetime
import json
import  re
from test1.items import taxwf
from scrapyluke.processors import *

class chinataxwf(scrapy.Spider):
    name = 'taxwf'
    count = 0
    start_urls = ['http://hd.chinatax.gov.cn/xxk/action/ListCategoryXml.do?pid=742']

    #起始url以及特殊页面的url
    def parse(self,response):
        a = re.findall(u'(\d{3})',response.body)
        yield scrapy.Request('http://hd.chinatax.gov.cn/xxk/action/ListXinxikucomXml.do?dotype=seriesarea&id=751,%E9%BB%91%E9%BE%99%E6%B1%9F5.15%E6%A1%88%E4%BB%B6',callback=self.parse_goto)
        yield scrapy.Request('http://hd.chinatax.gov.cn/xxk/action/ListXinxikucomXml.do?dotype=seriesarea&id=750,%E5%90%89%E6%9E%974.08%E6%A1%88%E4%BB%B6',callback=self.parse_goto)
        for s in a:
            yield scrapy.Request('http://hd.chinatax.gov.cn/xxk/action/ListXinxikucomXml.do?dotype=area&id=%s'%s,callback=self.parse_goto)

    def parse_goto(self,response):
        #获取各公司id，根据id发送get请求
        a =re.findall(u'(\d{6})',response.body)
        for s in a:
            yield scrapy.Request('http://hd.chinatax.gov.cn/xxk/action/GetArticleView1.do?op=xxkweb&id=%s'%s , callback=self.parse_info)

    def parse_info(self,response):
        taxwfinfo = taxwf()
        taxwfinfo['insert_time'] = str(datetime.datetime.now())
        taxwfinfo['url'] = response.url
        info = response.xpath('//body/table/tr/td/table')
        #处理两种不同的页面，并保存到item
        if '姓名' in info.xpath("tr[1]/td[1]/text()").extract_first():
            taxwfinfo['taxpayer_s_name'] = info.xpath("tr[1]/td[2]/text()").extract_first()
            taxwfinfo['sex'] = info.xpath("tr[2]/td[2]/text()").extract_first()
            taxwfinfo['representative_name__gender__name_and_number'] = info.xpath("tr[3]/td[2]/text()").extract_first()
            taxwfinfo['responsibility_information'] = info.xpath("tr[4]/td[2]/text()").extract_first()
            taxwfinfo['the_nature_of_the_case'] = info.xpath("tr[5]/td[2]/text()").extract_first()
            taxwfinfo['the_tax_treatment_of_punishment'] = info.xpath("tr[6]/td[2]/text()").extract_first()
        else:
            taxwfinfo['taxpayer_s_name'] = info.xpath("tr[1]/td[2]/text()").extract_first()
            taxwfinfo['taxpayer_identification_number'] = info.xpath("tr[2]/td[2]/text()").extract_first()
            taxwfinfo['organization_code'] = info.xpath("tr[3]/td[2]/text()").extract_first()
            taxwfinfo['registered_address'] = info.xpath("tr[4]/td[2]/text()").extract_first()
            taxwfinfo['representative_name__gender__name_and_number'] = info.xpath("tr[5]/td[2]/text()").extract_first()
            taxwfinfo['responsible_finance_name__gender__name_and_number'] = info.xpath("tr[6]/td[2]/text()").extract_first()
            taxwfinfo['responsibility_information'] = info.xpath("tr[7]/td[2]/text()").extract_first()
            taxwfinfo['the_nature_of_the_case'] = info.xpath("tr[8]/td[2]/text()").extract_first()
            taxwfinfo['the_tax_treatment_of_punishment'] = info.xpath("tr[9]/td[2]/text()").extract_first()
        yield taxwfinfo